# coding = utf-8
'''my words
v2
1、仅抓取书名，作者，出版社，价格，评级，封面等基础图书信息；
2、增加了书籍的类型以及标签；
'''

import requests
from bs4 import BeautifulSoup
import re
import time
import pdb

from bmob import *

b = Bmob("f5b3dbd027720a6f49c87b6a1a1c7a84", "616ee9149ae817bcd2f535b9a21050d4")



#爬取豆瓣所有的标签分类页面，并且提供每一个标签页面的URL
def provide_url():
    # 以http的get方式请求豆瓣页面（豆瓣的分类标签页面）
    responds = requests.get("https://book.douban.com/tag/?icn=index-nav")
    # html为获得响应的页面内容
    html = responds.text
    # 解析页面
    soup = BeautifulSoup(html, "lxml")
    # 获取类型区块
    book_area = soup.select("#content > div > .article > div > div")

    book_category = {}  # key为书籍类别，value为书籍tags地址
    for area in book_area:
        book_type = area.select("h2")[0].string[0:2]
        book_tag = area.select(".tagCol > tbody > tr > td > a")
        #pdb.set_trace()
        book_url_list = []
        for book in book_tag:
            book_url_list.append('https://book.douban.com/tag/' + str(book.string))
        book_category[book_type] = book_url_list
    return book_category

#获得评分人数的函数
def get_person(person):
    person = person.get_text().split()[0]
    person = re.findall(r'[0-9]+',person)
    return person

#当detail分为四段时候的获得价格函数
def get_rmb_price1(detail):
    price = detail.get_text().split('/',4)[-1].split()
    if re.match("USD", price[0]):
        price = float(price[1]) * 6
    elif re.match("CNY", price[0]):
        price = price[1]
    elif re.match("\A$", price[0]):
        price = float(price[1:len(price)]) * 6
    else:
        price = price[0]
    return price

#当detail分为三段时候的获得价格函数
def get_rmb_price2(detail):
    price = detail.get_text().split('/',3)[-1].split()
    if re.match("USD", price[0]):
        price = float(price[1]) * 6
    elif re.match("CNY", price[0]):
        price = price[1]
    elif re.match("\A$", price[0]):
        price = float(price[1:len(price)]) * 6
    else:
        price = price[0]
    return price

#测试输出函数
def test_print(name,author,intepretor,publish,time,price,score,person):
    print('name: ',name)
    print('author:', author)
    print('intepretor: ',intepretor)
    print('publish: ',publish)
    print('time: ',time)
    print('price: ',price)
    print('score: ',score)
    print('person: ',person)


def append_to_bmob(publish, rate, author, book_name, img_url, book_type, book_tag):
    # 插入一行数据，原子计数、Pointer
    print(
	b.insert(
		't_read', # 表名
		BmobUpdater.increment(
            "count", 2,
			{ # 书籍信息
				"publish": publish, #出版社
				"rate": rate, # 评分
				"author": author, # 作者
                "book_name" : book_name,  #书名
                "img_url" : img_url, #封面地址
                "book_type": book_type, #书籍类型
                "tag" : book_tag, #标签
			}
		)
	).jsonData # 输出json格式的内容
)


#解析每个页面获得其中需要信息的函数
def get_url_content(book_type, url):
    res = requests.get(url)
    #pdb.set_trace()
    html = res.text
    soup = BeautifulSoup(html.encode('utf-8'),"lxml")
    tag = url.split("?")[0].split("/")[-1]  #页面标签，就是页面链接中'tag/'后面的字符串
    titles = soup.select(".subject-list > .subject-item > .info > h2 > a") #包含书名的a标签
    details = soup.select(".subject-list > .subject-item > .info > .pub") #包含书的作者，出版社等信息的div标签
    scores = soup.select(".subject-list > .subject-item > .info > div > .rating_nums") #包含评分的span标签
    persons = soup.select(".subject-list > .subject-item > .info > div > .pl")  #包含评价人数的span标签
    image_urls = soup.select(".subject-list > .subject-item > .pic > a > img") #封面地址

    print("*******************这是 %s 类的书籍**********************" %tag)

    #打开文件，将信息写入文件
    file = open("./book_info.txt",'a') #可以更改为你自己的文件地址
    file.write("*******************这是 %s 类的书籍**********************" % tag)

    #用zip函数将相应的信息以元祖的形式组织在一起，以供后面遍历
    for title,detail,score,person,image_url in zip(titles,details,scores,persons,image_urls):
        try:#detail可以分成四段
            #pdb.set_trace()
            name = title.get_text().split()[0] #书名
            book_img = image_url.attrs['src']
            if detail.get_text().count('/') == 3:
                #'余华 / 作家出版社 / 2012-8-1 / 20.00元'
                author = detail.get_text().split('/', 3)[0].strip()  # 作者
                intepretor = "" # 译者
                publish = detail.get_text().split('/', 3)[1].strip()  # 出版社
                time = detail.get_text().split('/', 3)[2].strip()  # 出版年份
            else:
                #'[日] 东野圭吾 / 李盈春 / 南海出版公司 / 2014-5 / 39.50元'
                author = detail.get_text().split('/', 4)[0].strip()  # 作者
                intepretor = detail.get_text().split('/', 4)[1].strip() # 译者
                publish = detail.get_text().split('/', 4)[2].strip()  # 出版社
                time = detail.get_text().split('/', 4)[3].strip() # 出版年份
            price = get_rmb_price1(detail)   #获取价格
            score = score.get_text() if True else ""   #如果没有评分就置空
            person = get_person(person)  #获得评分人数
            #在控制台测试打印
            test_print(name,author,intepretor,publish,time,price,score,person)
            append_to_bmob(publish, score, author, name, book_img,book_type, tag)
            #将书籍信息写入txt文件
            try:
                file.write('name: %s ' % name)
                file.write('author: %s ' % author)
                file.write('intepretor: %s ' % intepretor)
                file.write('publish: %s ' % publish)
                file.write('time: %s ' % time)
                file.write('price: %s ' % price)
                file.write('score: %s ' % score)
                file.write('person: %s ' % person)
                file.write('\n')
            except (IndentationError,UnicodeEncodeError):
                continue

        except IndexError:
            try:#detail可以分成三段
                name = title.get_text().split()[0]  # 书名
                if detail.get_text().count('/') == 3:
                    #'余华 / 作家出版社 / 2012-8-1 / 20.00元'
                    author = detail.get_text().split('/', 3)[0].split()[0]  # 作者
                    intepretor = "" # 译者
                    publish = detail.get_text().split('/', 3)[1].strip()  # 出版社
                    time = detail.get_text().split('/', 3)[2].strip()  # 出版年份
                else:
                    #'[日] 东野圭吾 / 李盈春 / 南海出版公司 / 2014-5 / 39.50元'
                    author = detail.get_text().split('/', 4)[0].split()[0]  # 作者
                    intepretor = detail.get_text().split('/', 4)[1].strip() # 译者
                    publish = detail.get_text().split('/', 4)[2].strip()  # 出版社
                    time = detail.get_text().split('/', 4)[3].strip() # 出版年份
                price = get_rmb_price2(detail)  # 获取价格
                score = score.get_text() if True else ""  # 如果没有评分就置空
                person = get_person(person)  # 获得评分人数
                #在控制台测试打印
                test_print(name, author, intepretor, publish, time, price, score, person)
                append_to_bmob(publish, score, author, name, book_img)
                #将书籍信息写入txt文件
                try:
                    file.write('name: %s ' % name)
                    file.write('author: %s ' % author)
                    file.write('intepretor: %s ' % intepretor)
                    file.write('publish: %s ' % publish)
                    file.write('time: %s ' % time)
                    file.write('price: %s ' % price)
                    file.write('score: %s ' % score)
                    file.write('person: %s ' % person)
                    file.write('\n')
                except (IndentationError, UnicodeEncodeError):
                    continue

            except (IndexError,TypeError):
                continue

        except TypeError:
            continue
    file

    file.write('\n')
    file.close()  #关闭文件


#程序执行入口
if __name__ == '__main__':
    #url = "https://book.douban.com/tag/程序"
    book_url_dict = provide_url()
    for dct in book_url_dict:
        book_type = dct
        book_urls = book_url_dict[dct]
        for url in book_urls:
            get_url_content(book_type, url)  #解析每一个URL的内容
            #time.sleep(5000)